/**
 * 所有爬虫的基础类，继承该类，并重写getItemWithPage方法即可 
 */
var q = require("q");
var cheerio = require("cheerio");

var base = function () {
    this.version = "1.0.0";
    this.name = "base spider";
    this.enabled = true;
    this.page = 5;
    this.items = [];
    this.encode = "utf-8";
    this.contentType = "html"; // html, text, json
};

base.prototype.getUrls = function (spider) {
    var d = q.defer();
    var result = q('');
    var self = this;
    console.log(self.name + " " + self.version + " start...");
    console.log("---------------------------------------");
    for (var i = 1; i <= self.page; i++) {
        result = result.then(self.getItemWithPage(spider, i));
    }
    result.then(function () {
        d.resolve(self.items);
        console.log(self.name + " " + self.version + " finished!");
    });
    return d.promise;
};

// 在一般爬虫的编写中，重写这个方法，返回指定页码的url地址
base.prototype.getPageUrl = function (pageIndex) {
    return "";
};

// 在一般爬虫的编写中，重写这个方法，返回指定页面的item集合
base.prototype.getItems = function ($) {
    var items = [];
    // $("ul li a").each(function (index, ele) {
    //     var e = $(ele);
    //     var item = {};
    //     item.title = e.text();
    //     item.url = "http://www.ahzfcg.gov.cn/" + e.attr("href");
    //     item.source = ah_spider.name;
    //     items.push(item);
    // });
    return items;
};

base.prototype.getItemWithPage = function (spider, pageIndex) {
    var self = this;
    return function () {
        console.log("process page: " + pageIndex);
        var d = q.defer();
        var url = self.getPageUrl(pageIndex);
        if (url == "") {
            console.error("Page URL is invalid.");
            setTimeout(function () {
                d.resolve();
            }, 0);
        } else {
            spider.encode = self.encode;
            spider.get(url, function (data, err) {
                if (err || data == "") {
                    d.resolve();
                } else {
                    try {
                        var items;
                        if (self.contentType == "html") {
                            var $ = cheerio.load(data);
                            items = self.getItems($);
                        } else if (self.contentType == "json") {
                            var jsonObj = eval('(' + data + ')');
                            items = self.getItems(jsonObj);
                        } else if (self.contentType == "text") {
                            items = self.getItems(data);
                        }
                        // array copy
                        if (items && items.length > 0) {
                            for (var i = 0, j = items.length; i < j; i++) {
                                self.items.push(items[i]);
                            }
                        }
                        d.resolve();
                    } catch (e) {
                        console.error(e);
                        d.resolve();
                    }
                }
            });
        }
        return d.promise;
    };
};

exports = module.exports = base;